{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import os\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils import data\n",
    "\n",
    "\n",
    "class LyricsNGramsDataset(data.Dataset):\n",
    "    file_dir = Path().cwd()\n",
    "    base_dir = file_dir.parents[0]\n",
    "    \n",
    "#     filepath = Path(__file__).absolute()\n",
    "#     base_dir = filepath.parents[1]\n",
    "    data_dir = base_dir / 'data' / 'raw' / 'lmd-full_and_reddit_MIDI_dataset'\n",
    "    syllable_level_dir = data_dir / 'syllable_level_npy_39'\n",
    "\n",
    "    def __init__(self, ngram=3):\n",
    "        # initialize the dataset by creating SkipGrap Data\n",
    "        # 1. Load in all the files\n",
    "        # 2. Fetch out the lyrics from it\n",
    "        # 3. Create n-grams as required\n",
    "        f_names = self.syllable_level_dir.iterdir()\n",
    "        vocab = set()\n",
    "        ngrams = []\n",
    "        for i, f_name in enumerate(f_names):\n",
    "            f_data = np.load(f_name, allow_pickle=True)\n",
    "            lyrics = f_data[0][2]\n",
    "#             lyrics = lyrics[:100]\n",
    "            f_ngrams = self.generate_ngrams(lyrics, ngram)\n",
    "            ngrams.extend(f_ngrams)\n",
    "            vocab = vocab.union(lyrics)\n",
    "            if i==5:\n",
    "                break\n",
    "\n",
    "        self.ngrams = ngrams\n",
    "        self.vocab = vocab\n",
    "        self.vocab_size = len(self.vocab)\n",
    "        self.word_to_ix = {word: i for i, word in enumerate(vocab)}\n",
    "\n",
    "        idx_ngrams = [[self.word_to_ix[w] for w in ngram] for ngram in ngrams]\n",
    "        self.idx_ngrams = [[ngram[:-1], ngram[-1]] for ngram in idx_ngrams]\n",
    "\n",
    "    def generate_ngrams(self, word_lst, n):\n",
    "        # Use the zip function to help us generate n-grams\n",
    "        # Return a list of tuples\n",
    "        # Each tuple is (word_i-2, word_i-1, word_i)\n",
    "        ngrams = zip(*[word_lst[i:] for i in range(n)])\n",
    "        return [ngram for ngram in ngrams]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.idx_ngrams)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        context, target = self.idx_ngrams[i]\n",
    "        context = torch.tensor(context, dtype=torch.long)\n",
    "        target = torch.tensor(target, dtype=torch.long)\n",
    "        return context, target\n",
    "\n",
    "\n",
    "class LyricsEmbeddings(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, context_size, hidden_dim=128):\n",
    "        super(LyricsEmbeddings, self).__init__()\n",
    "\n",
    "        # matrix to keep the embeddings\n",
    "        self.embeddings = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.linear1 = nn.Linear(context_size * embedding_dim, hidden_dim)\n",
    "        self.linear2 = nn.Linear(hidden_dim, vocab_size)\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        # print(inputs)\n",
    "        # check why this view is needed!\n",
    "        embeds = self.embeddings(inputs)\n",
    "        embeds_shape = embeds.shape[0]\n",
    "        # print(embeds)\n",
    "        # print(embeds.shape)\n",
    "        embeds = embeds.view((embeds_shape, -1))\n",
    "        out = F.relu(self.linear1(embeds))\n",
    "        out = self.linear2(out)\n",
    "\n",
    "        log_probab = F.log_softmax(out, dim=1)\n",
    "#         print(log_probab)\n",
    "#         print(log_probab.shape)\n",
    "        return log_probab\n",
    "\n",
    "\n",
    "class LossCompute(object):\n",
    "    def __init__(self):\n",
    "        self.criterion = nn.NLLLoss()\n",
    "\n",
    "    def __call__(self, x, y):\n",
    "        \"\"\"\n",
    "        Call to compute loss\n",
    "        :param x: predicted value\n",
    "        :param y: actual value\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        loss = self.criterion(x, y)\n",
    "        return loss\n",
    "\n",
    "\n",
    "def train(train_data_iterator, model, optimizer, criterion, epochs, device):\n",
    "    losses = []\n",
    "    for epoch in range(epochs):\n",
    "        model.train()\n",
    "        print(\"Running epoch {} / {}\".format(epoch+1, epochs))\n",
    "        total_loss = 0\n",
    "\n",
    "        for num_steps, data in enumerate(train_data_iterator):\n",
    "            context = data[0].to(device)\n",
    "            target = data[1].to(device)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # print(context)\n",
    "            log_probabs = model(context)\n",
    "\n",
    "            loss = criterion(log_probabs, target)\n",
    "            # print(loss)\n",
    "            # print(type(loss))\n",
    "            print(\"Before\")\n",
    "            print(list(model.parameters())[0].grad)\n",
    "#             a = list(model.parameters())[0].clone()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            print(\"After\")\n",
    "            print(list(model.parameters())[0].grad)\n",
    "#             b = list(model.parameters())[0].clone()\n",
    "#             print(torch.equal(a.data, b.data))\n",
    "\n",
    "            total_loss += loss.item()\n",
    "\n",
    "#             if num_steps == 1:\n",
    "#                 break\n",
    "\n",
    "        losses.append(total_loss)\n",
    "    print(losses)\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n"
     ]
    }
   ],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device('cuda:0' if use_cuda else 'cpu')\n",
    "print(\"Using {} device\".format(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "# Dataloader params\n",
    "data_params = {'batch_size': 1000,\n",
    "               'shuffle': True,\n",
    "               'num_workers': 1}\n",
    "\n",
    "# Model params\n",
    "ngrams = 3\n",
    "context_size = ngrams - 1\n",
    "embedding_dim = 12\n",
    "hidden_dim = 12\n",
    "\n",
    "# Training params\n",
    "epochs = 10\n",
    "learning_rate = 0.1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set = LyricsNGramsDataset(ngrams)\n",
    "train_data_iterator = data.DataLoader(training_set, **data_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size is: 550\n"
     ]
    }
   ],
   "source": [
    "vocab_size = training_set.vocab_size\n",
    "print(\"Vocabulary size is: {}\".format(vocab_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LyricsEmbeddings(vocab_size, embedding_dim, context_size, hidden_dim)\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = list(model.parameters())[0].clone()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_weights = model.embeddings.weight.data\n",
    "layer1_weights = model.linear1.weight.data\n",
    "layer2_weights = model.linear2.weight.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# layer1_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# layer2_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LyricsEmbeddings(\n",
      "  (embeddings): Embedding(550, 12)\n",
      "  (linear1): Linear(in_features=24, out_features=12, bias=True)\n",
      "  (linear2): Linear(in_features=12, out_features=550, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "criterion = LossCompute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running epoch 1 / 10\n",
      "Before\n",
      "None\n",
      "After\n",
      "tensor([[-1.4551e-05, -1.4094e-05,  2.4908e-05,  ..., -2.4296e-05,\n",
      "          2.2394e-07,  6.6265e-06],\n",
      "        [-2.4087e-05,  3.7635e-05,  5.2059e-05,  ...,  1.4019e-05,\n",
      "         -4.4286e-05,  1.4824e-05],\n",
      "        [ 4.5352e-05, -3.2851e-05, -8.5730e-05,  ...,  1.7971e-05,\n",
      "         -3.3788e-05, -1.0455e-05],\n",
      "        ...,\n",
      "        [ 2.3383e-04,  1.5947e-04, -1.9655e-04,  ..., -1.0594e-04,\n",
      "          5.4424e-05, -1.8465e-04],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 6.4759e-05,  1.1012e-04,  8.2552e-05,  ..., -4.6484e-05,\n",
      "         -2.3722e-05,  3.6635e-05],\n",
      "        [-8.1672e-07, -4.9748e-05,  2.2166e-05,  ..., -1.1560e-05,\n",
      "          3.0478e-05,  1.1506e-07],\n",
      "        [-4.9920e-05, -6.2705e-05,  1.4971e-04,  ..., -1.1860e-04,\n",
      "          1.2234e-04, -8.1930e-05],\n",
      "        ...,\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-5.6549e-06, -1.0946e-05,  4.9767e-05,  ..., -5.3003e-05,\n",
      "          4.3206e-06, -9.2327e-05],\n",
      "        [ 2.1793e-05, -3.2261e-05, -1.0810e-04,  ...,  5.7674e-05,\n",
      "          1.7731e-05,  4.8694e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 6.8115e-05,  8.9726e-05,  1.3468e-04,  ..., -1.3247e-04,\n",
      "          1.4990e-04,  8.5349e-05],\n",
      "        [-4.9797e-05,  8.2151e-05, -1.3800e-04,  ...,  2.4433e-04,\n",
      "         -4.1076e-05, -4.1466e-05],\n",
      "        ...,\n",
      "        [-4.6685e-05, -3.6628e-04, -1.2851e-04,  ...,  2.0376e-04,\n",
      "         -5.5995e-05,  4.8898e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Running epoch 2 / 10\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 3.1515e-05,  4.2096e-05,  6.3021e-05,  ..., -6.1685e-05,\n",
      "          7.0199e-05,  3.9853e-05],\n",
      "        [-3.1894e-05, -1.0609e-04, -1.3376e-04,  ..., -1.9942e-05,\n",
      "          2.7175e-05, -1.1408e-04],\n",
      "        ...,\n",
      "        [ 9.3489e-05, -1.2009e-04, -1.9598e-04,  ...,  1.8248e-05,\n",
      "          2.4644e-05, -1.4357e-04],\n",
      "        [ 6.7787e-06,  8.3709e-06,  8.1513e-06,  ..., -1.6665e-05,\n",
      "          3.0630e-06, -2.1059e-07],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[-1.4305e-05, -1.4065e-05,  2.4813e-05,  ..., -2.3995e-05,\n",
      "          3.3403e-07,  5.9420e-06],\n",
      "        [-2.3843e-05, -8.7372e-06,  7.4666e-05,  ...,  9.9138e-07,\n",
      "         -1.2670e-05,  1.5428e-05],\n",
      "        [ 5.4432e-05,  1.1323e-04, -1.4512e-05,  ...,  1.5019e-04,\n",
      "         -8.4451e-05,  8.7122e-05],\n",
      "        ...,\n",
      "        [ 3.0194e-05,  2.4614e-05,  4.7641e-06,  ..., -3.3594e-06,\n",
      "          1.0193e-05,  1.5433e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-1.6591e-06,  6.9389e-06, -5.4838e-05,  ..., -1.3100e-05,\n",
      "          2.1250e-05, -1.0232e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 1.3948e-04,  2.3471e-04,  1.7742e-04,  ..., -9.8897e-05,\n",
      "         -4.9003e-05,  7.8336e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-1.1034e-04, -1.3083e-04,  3.1759e-04,  ..., -2.4958e-04,\n",
      "          2.5327e-04, -1.6969e-04],\n",
      "        ...,\n",
      "        [ 2.7400e-04,  1.2890e-04, -1.7769e-04,  ..., -1.1778e-04,\n",
      "          1.2627e-05, -1.2569e-04],\n",
      "        [-2.9262e-05, -4.3042e-05,  8.6774e-05,  ..., -7.9077e-05,\n",
      "          1.1958e-06, -1.9450e-04],\n",
      "        [ 4.9255e-05, -8.4715e-05, -1.1389e-04,  ...,  1.4829e-04,\n",
      "         -6.2342e-06,  1.2669e-04]])\n",
      "Running epoch 3 / 10\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 1.4199e-05,  2.7780e-05,  2.1009e-05,  ...,  2.3244e-05,\n",
      "          8.6397e-06,  2.6561e-05],\n",
      "        [ 1.0035e-05, -1.0682e-04, -5.2368e-05,  ..., -4.9503e-05,\n",
      "          2.5016e-05, -1.4144e-04],\n",
      "        ...,\n",
      "        [ 7.5617e-05, -1.4353e-05, -1.8397e-04,  ..., -3.5178e-05,\n",
      "          2.2436e-05, -1.6369e-04],\n",
      "        [-1.3824e-05, -2.0610e-05,  4.0130e-05,  ..., -3.7227e-05,\n",
      "          6.1179e-07, -9.0548e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 6.4883e-05,  1.0943e-04,  8.2565e-05,  ..., -4.5699e-05,\n",
      "         -2.2432e-05,  3.7024e-05],\n",
      "        [-3.8102e-05, -3.4247e-05,  5.3226e-05,  ..., -2.3069e-05,\n",
      "         -2.0868e-05, -1.0087e-05],\n",
      "        [ 7.9234e-06,  1.3215e-05,  6.3377e-05,  ..., -1.2411e-05,\n",
      "          2.4448e-05,  2.7589e-05],\n",
      "        ...,\n",
      "        [ 3.0366e-05,  2.5723e-05,  5.0738e-06,  ..., -3.0595e-06,\n",
      "          1.1383e-05,  1.3592e-05],\n",
      "        [ 7.0170e-06,  8.0963e-06,  7.0767e-06,  ..., -1.6717e-05,\n",
      "          2.8258e-06, -1.6595e-08],\n",
      "        [ 2.1706e-05, -3.2801e-05, -1.0797e-04,  ...,  5.5044e-05,\n",
      "          1.9089e-05,  4.9217e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[-3.0320e-05, -2.9495e-05,  5.2947e-05,  ..., -5.0439e-05,\n",
      "          1.0206e-06,  1.1423e-05],\n",
      "        [ 6.5282e-05,  9.2937e-05,  1.3710e-04,  ..., -1.2794e-04,\n",
      "          1.5219e-04,  8.5162e-05],\n",
      "        [ 2.8001e-05,  2.0449e-04,  7.9950e-05,  ...,  4.7238e-05,\n",
      "         -2.5730e-06, -4.1098e-05],\n",
      "        ...,\n",
      "        [ 3.1394e-04, -9.7837e-05, -1.9828e-04,  ..., -1.2271e-05,\n",
      "          2.0116e-05, -9.6801e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Running epoch 4 / 10\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 2.7795e-05, -4.5115e-06,  8.7264e-05,  ..., -7.1446e-05,\n",
      "          1.0126e-04,  3.9261e-05],\n",
      "        [ 2.9288e-05,  2.7070e-05,  3.2166e-05,  ..., -7.8431e-05,\n",
      "          2.1343e-05, -7.3840e-05],\n",
      "        ...,\n",
      "        [ 9.0341e-05, -1.0099e-04, -1.9168e-04,  ...,  9.7109e-06,\n",
      "          2.0334e-05, -1.5707e-04],\n",
      "        [-7.8335e-06, -1.3221e-05,  4.6063e-05,  ..., -5.4498e-05,\n",
      "          2.7517e-06, -8.9435e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[-1.4156e-05, -1.3697e-05,  2.4643e-05,  ..., -2.3126e-05,\n",
      "          5.1676e-07,  5.0479e-06],\n",
      "        [-3.7630e-05, -3.2173e-05,  5.3323e-05,  ..., -2.3101e-05,\n",
      "         -2.0017e-05, -9.0498e-06],\n",
      "        [-4.9587e-05, -8.7027e-05, -3.4582e-05,  ...,  3.5557e-05,\n",
      "          1.7596e-05,  6.7748e-05],\n",
      "        ...,\n",
      "        [ 5.1309e-05, -5.1563e-05, -2.3910e-05,  ...,  1.0866e-05,\n",
      "         -2.6181e-05,  2.1684e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 5.8356e-06, -5.7447e-05, -1.2022e-04,  ...,  4.6340e-05,\n",
      "          1.9306e-05,  4.8818e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 1.3922e-04,  2.3269e-04,  1.7581e-04,  ..., -9.7605e-05,\n",
      "         -4.6629e-05,  7.9002e-05],\n",
      "        [ 3.5791e-05,  1.6187e-04, -6.3462e-06,  ...,  7.2115e-05,\n",
      "         -4.4950e-05,  5.5789e-05],\n",
      "        [ 1.0517e-04,  1.3656e-04,  1.1377e-04,  ...,  5.0513e-06,\n",
      "         -2.5387e-06, -2.5816e-04],\n",
      "        ...,\n",
      "        [ 2.3761e-04,  2.5323e-04, -1.1000e-04,  ..., -1.4193e-04,\n",
      "          1.1957e-04, -1.4672e-04],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running epoch 5 / 10\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[-1.4073e-05, -1.3440e-05,  2.4701e-05,  ..., -2.2932e-05,\n",
      "          5.1775e-07,  4.9435e-06],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-1.7379e-05, -1.5024e-04, -4.3022e-05,  ..., -1.4209e-04,\n",
      "          5.3690e-05, -1.5441e-04],\n",
      "        ...,\n",
      "        [ 2.7789e-05, -1.2419e-04, -9.7510e-05,  ...,  6.0803e-05,\n",
      "          4.3312e-05, -6.6533e-05],\n",
      "        [ 6.9094e-06,  7.9246e-06,  5.6278e-06,  ..., -1.6553e-05,\n",
      "          2.2085e-06,  1.6257e-07],\n",
      "        [ 5.9395e-06, -5.7821e-05, -1.2058e-04,  ...,  4.5842e-05,\n",
      "          2.0101e-05,  4.8666e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-3.2223e-07,  1.3946e-04,  1.1444e-04,  ..., -7.4099e-05,\n",
      "          2.3499e-05,  2.4181e-05],\n",
      "        [ 1.0492e-04,  3.3663e-05, -5.6860e-05,  ..., -7.4547e-05,\n",
      "         -4.8747e-05, -2.1551e-05],\n",
      "        ...,\n",
      "        [ 2.2547e-04,  8.8879e-05, -1.6637e-04,  ..., -1.0994e-04,\n",
      "          1.0135e-05, -1.4500e-04],\n",
      "        [-1.5838e-05, -2.2165e-05,  3.8577e-05,  ..., -3.8593e-05,\n",
      "         -3.1121e-07, -8.8055e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 1.3856e-04,  2.3128e-04,  1.7652e-04,  ..., -9.6661e-05,\n",
      "         -4.5068e-05,  7.9054e-05],\n",
      "        [-7.8684e-06, -9.9517e-05,  4.9273e-05,  ..., -2.3859e-05,\n",
      "          6.2780e-05, -1.0217e-06],\n",
      "        [-1.2614e-04,  2.5992e-04,  3.2603e-04,  ...,  3.7321e-04,\n",
      "          6.2192e-05,  1.1272e-04],\n",
      "        ...,\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Running epoch 6 / 10\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 5.0650e-05,  9.4514e-05,  1.0698e-04,  ..., -6.7671e-05,\n",
      "         -2.0197e-05,  4.1581e-05],\n",
      "        [-4.0875e-05, -7.5214e-05,  7.7226e-05,  ..., -3.4853e-05,\n",
      "          9.9967e-06, -7.7951e-06],\n",
      "        [-1.3720e-04, -1.2855e-04,  1.5761e-04,  ..., -8.6763e-05,\n",
      "          1.7859e-04,  1.9524e-05],\n",
      "        ...,\n",
      "        [-2.6492e-06, -1.4995e-04, -1.0234e-04,  ...,  6.0457e-05,\n",
      "          2.9810e-05, -7.9117e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 3.7815e-05,  1.6987e-04,  6.0213e-05,  ..., -4.9204e-05,\n",
      "          4.4135e-05,  3.1604e-05],\n",
      "        [ 1.1881e-04,  3.9768e-05, -5.7644e-05,  ..., -1.5313e-04,\n",
      "         -3.6698e-05, -1.8818e-04],\n",
      "        ...,\n",
      "        [ 2.2548e-04,  8.5839e-05, -1.6484e-04,  ..., -1.1178e-04,\n",
      "          1.4835e-05, -1.4837e-04],\n",
      "        [-1.7060e-05, -2.2349e-05,  3.8313e-05,  ..., -3.9094e-05,\n",
      "         -9.2605e-07, -8.7451e-05],\n",
      "        [ 2.2389e-05, -4.1716e-05, -5.2123e-05,  ...,  6.5949e-05,\n",
      "         -3.3380e-06,  5.9333e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 9.6926e-05,  2.0225e-04, -9.6076e-05,  ...,  4.2117e-04,\n",
      "         -2.5175e-04,  1.0795e-04],\n",
      "        ...,\n",
      "        [ 6.7055e-05,  6.0742e-05,  1.3579e-05,  ..., -2.2263e-06,\n",
      "          3.2564e-05,  1.8156e-05],\n",
      "        [ 1.4771e-05,  1.6600e-05,  8.7907e-06,  ..., -3.5447e-05,\n",
      "          3.7460e-06,  1.0116e-06],\n",
      "        [-3.5799e-05, -3.4005e-05, -1.4755e-04,  ..., -4.6088e-05,\n",
      "          5.1281e-05, -2.1304e-05]])\n",
      "Running epoch 7 / 10\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[-1.3771e-05, -1.3373e-05,  2.4861e-05,  ..., -2.2322e-05,\n",
      "          8.9442e-07,  4.0395e-06],\n",
      "        [ 3.7531e-05,  1.7026e-04,  5.9832e-05,  ..., -4.8241e-05,\n",
      "          4.4828e-05,  3.1165e-05],\n",
      "        [ 2.5887e-05,  1.6031e-04,  1.5021e-04,  ..., -6.8274e-05,\n",
      "          5.9114e-05, -9.2405e-05],\n",
      "        ...,\n",
      "        [ 9.7164e-05,  2.7753e-05, -8.5235e-05,  ..., -5.5657e-05,\n",
      "          9.7140e-07, -7.9548e-05],\n",
      "        [ 6.9673e-06,  7.5513e-06,  3.9484e-06,  ..., -1.6447e-05,\n",
      "          1.6564e-06,  4.2722e-07],\n",
      "        [-1.6717e-05, -1.5796e-05, -6.8812e-05,  ..., -2.1689e-05,\n",
      "          2.4062e-05, -9.6393e-06]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-3.6119e-05, -2.7067e-05,  5.5607e-05,  ..., -2.4421e-05,\n",
      "         -1.8618e-05, -5.9775e-06],\n",
      "        [ 8.3537e-05, -1.9147e-05, -1.3994e-04,  ...,  1.5844e-04,\n",
      "         -1.5766e-04,  2.8838e-05],\n",
      "        ...,\n",
      "        [ 1.2964e-04,  3.8877e-05, -7.6485e-05,  ..., -5.5001e-05,\n",
      "          2.3779e-05, -6.6740e-05],\n",
      "        [-1.7930e-05, -2.3002e-05,  3.7933e-05,  ..., -3.9402e-05,\n",
      "         -1.6281e-06, -8.6313e-05],\n",
      "        [ 2.1776e-05, -4.2427e-05, -5.1779e-05,  ...,  6.4871e-05,\n",
      "         -3.4678e-06,  5.9518e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 1.3820e-04,  2.2938e-04,  1.7594e-04,  ..., -9.6702e-05,\n",
      "         -4.2134e-05,  7.8937e-05],\n",
      "        [-1.0130e-05, -9.5819e-05,  4.9351e-05,  ..., -2.3087e-05,\n",
      "          6.1438e-05, -1.3425e-06],\n",
      "        [-1.6787e-04, -2.7388e-04,  9.2455e-05,  ..., -2.7748e-04,\n",
      "          2.2807e-04, -8.4680e-05],\n",
      "        ...,\n",
      "        [ 6.1848e-05, -2.1765e-04, -2.0799e-04,  ...,  1.1755e-04,\n",
      "          8.3825e-05, -1.7457e-04],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Running epoch 8 / 10\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-1.5943e-05,  6.9344e-05,  1.1999e-04,  ..., -1.0367e-04,\n",
      "          4.7915e-05,  5.6686e-07],\n",
      "        [-4.6910e-05, -9.9335e-05, -5.4150e-05,  ..., -2.4078e-05,\n",
      "          1.5441e-05, -3.6981e-06],\n",
      "        ...,\n",
      "        [ 1.2686e-04, -9.1916e-05, -1.7917e-04,  ..., -8.0015e-07,\n",
      "          4.7942e-05, -1.5778e-04],\n",
      "        [-1.1343e-05, -1.5813e-05,  4.0666e-05,  ..., -5.6185e-05,\n",
      "         -6.5674e-07, -8.5418e-05],\n",
      "        [-1.6731e-05, -1.5371e-05, -6.9260e-05,  ..., -2.1855e-05,\n",
      "          2.4437e-05, -9.1751e-06]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 5.0731e-05,  9.3628e-05,  1.0667e-04,  ..., -6.6659e-05,\n",
      "         -1.7984e-05,  4.0699e-05],\n",
      "        [ 1.2516e-05,  3.0863e-05,  1.8413e-05,  ...,  2.1076e-05,\n",
      "          8.5132e-06,  2.4508e-05],\n",
      "        [-1.5903e-05, -5.4931e-05,  1.1181e-04,  ..., -6.7989e-05,\n",
      "          4.9062e-05, -4.9672e-06],\n",
      "        ...,\n",
      "        [ 9.7047e-05,  2.5595e-05, -8.4027e-05,  ..., -5.6656e-05,\n",
      "          2.7454e-06, -8.1098e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 2.1786e-05, -4.3308e-05, -5.1541e-05,  ...,  6.4076e-05,\n",
      "         -3.5991e-06,  5.9646e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-1.6234e-05,  7.4649e-05,  1.4204e-04,  ...,  1.9026e-04,\n",
      "         -2.6820e-05,  7.6538e-05],\n",
      "        ...,\n",
      "        [ 6.8774e-05,  6.4462e-05,  1.4102e-05,  ..., -3.3927e-07,\n",
      "          3.6838e-05,  1.2415e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Running epoch 9 / 10\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-1.7969e-05,  5.0496e-05,  5.0859e-05,  ...,  6.9013e-06,\n",
      "         -3.7128e-05,  2.0631e-05],\n",
      "        [ 2.3065e-05,  2.4639e-05,  3.4672e-05,  ...,  8.8256e-05,\n",
      "         -4.1656e-05,  1.3388e-05],\n",
      "        ...,\n",
      "        [ 2.0833e-04, -7.8771e-05, -2.0991e-04,  ..., -3.7465e-05,\n",
      "         -1.2013e-06, -1.4180e-04],\n",
      "        [ 7.0358e-06,  7.2991e-06,  2.2880e-06,  ..., -1.6244e-05,\n",
      "          1.0235e-06,  7.4572e-07],\n",
      "        [-1.6601e-05, -1.5055e-05, -6.9877e-05,  ..., -2.2009e-05,\n",
      "          2.4780e-05, -8.7841e-06]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[-1.3482e-05, -1.3146e-05,  2.4915e-05,  ..., -2.1600e-05,\n",
      "          1.2263e-06,  3.0296e-06],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-8.7326e-05, -2.0843e-04,  8.1869e-05,  ..., -2.0695e-04,\n",
      "          9.2428e-05, -5.1326e-05],\n",
      "        ...,\n",
      "        [ 9.7681e-05,  7.7559e-06, -6.9878e-05,  ..., -1.6396e-05,\n",
      "          3.9596e-05, -8.2906e-05],\n",
      "        [-1.9695e-05, -2.3951e-05,  3.6791e-05,  ..., -4.0415e-05,\n",
      "         -2.5581e-06, -8.4733e-05],\n",
      "        [ 2.1741e-05, -4.4087e-05, -5.1424e-05,  ...,  6.3309e-05,\n",
      "         -3.7340e-06,  5.9555e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 1.3779e-04,  2.2867e-04,  1.7571e-04,  ..., -9.5335e-05,\n",
      "         -4.0138e-05,  8.0487e-05],\n",
      "        [ 3.0533e-05,  1.1357e-04,  1.8670e-04,  ..., -1.8725e-04,\n",
      "          2.0391e-04,  1.0765e-05],\n",
      "        [-2.4686e-05,  1.5120e-04,  2.9346e-05,  ...,  2.0056e-04,\n",
      "         -3.9855e-05,  1.1431e-04],\n",
      "        ...,\n",
      "        [-1.0646e-04,  7.3306e-05,  5.1704e-05,  ..., -1.3536e-05,\n",
      "          7.2209e-05, -3.3224e-05],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Running epoch 10 / 10\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[-1.3451e-05, -1.3287e-05,  2.4894e-05,  ..., -2.1423e-05,\n",
      "          1.5341e-06,  2.5774e-06],\n",
      "        [-5.9869e-06, -4.3452e-05,  2.3312e-05,  ..., -1.0326e-05,\n",
      "          2.8270e-05, -9.6792e-07],\n",
      "        [ 1.9511e-05,  5.4898e-05,  3.0061e-05,  ...,  1.2884e-04,\n",
      "         -7.5661e-05,  4.3642e-05],\n",
      "        ...,\n",
      "        [ 1.2945e-04,  5.4317e-05, -7.6529e-05,  ..., -5.7543e-05,\n",
      "          2.4138e-05, -7.9257e-05],\n",
      "        [ 7.1342e-06,  7.1970e-06,  1.5166e-06,  ..., -1.6215e-05,\n",
      "          6.8887e-07,  1.0071e-06],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [-1.7478e-05,  5.1924e-05,  5.0846e-05,  ...,  5.8397e-06,\n",
      "         -3.6204e-05,  2.1058e-05],\n",
      "        [ 1.8127e-06, -6.4729e-05, -6.6347e-05,  ...,  4.3796e-05,\n",
      "         -7.5131e-05,  4.7683e-05],\n",
      "        ...,\n",
      "        [ 9.5590e-05, -1.2267e-04, -1.8273e-04,  ..., -6.4640e-06,\n",
      "          3.5055e-05, -1.7232e-04],\n",
      "        [-2.0498e-05, -2.4367e-05,  3.6471e-05,  ..., -4.1277e-05,\n",
      "         -2.9653e-06, -8.4175e-05],\n",
      "        [ 4.8262e-06, -5.9255e-05, -1.2163e-04,  ...,  4.0405e-05,\n",
      "          2.1074e-05,  5.1704e-05]])\n",
      "Before\n",
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "After\n",
      "tensor([[ 1.3765e-04,  2.2677e-04,  1.7520e-04,  ..., -9.4527e-05,\n",
      "         -3.8809e-05,  7.9906e-05],\n",
      "        [ 4.4366e-05,  2.0772e-04,  1.3677e-04,  ..., -1.6160e-04,\n",
      "          1.4665e-04,  1.0789e-05],\n",
      "        [-2.0921e-04, -2.1709e-04,  3.6061e-04,  ..., -4.2780e-04,\n",
      "          3.7164e-04, -1.5789e-04],\n",
      "        ...,\n",
      "        [ 6.9376e-05,  6.7798e-05,  1.4567e-05,  ...,  1.2051e-06,\n",
      "          4.0976e-05,  6.8099e-06],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]])\n",
      "[19.06545352935791, 19.032399654388428, 19.018486976623535, 19.003929138183594, 18.995147705078125, 18.961827754974365, 18.94815492630005, 18.932884216308594, 18.909008026123047, 18.884950160980225]\n"
     ]
    }
   ],
   "source": [
    "trained_model = train(train_data_iterator, model, optimizer, criterion, epochs, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = list(model.parameters())[0].clone()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[False, False, False,  ..., False, False, False],\n",
       "        [False, False, False,  ..., False, False, False],\n",
       "        [False, False, False,  ..., False, False, False],\n",
       "        ...,\n",
       "        [False, False, False,  ..., False, False, False],\n",
       "        [False, False, False,  ..., False, False, False],\n",
       "        [False, False, False,  ..., False, False, False]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.eq(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.8676, -0.2912,  0.4140,  ..., -0.4085, -0.4169,  0.8158],\n",
       "        [-1.4329, -0.4226,  0.9353,  ..., -0.1001, -0.6380,  0.4034],\n",
       "        [-0.0900,  0.2997, -0.0790,  ...,  0.8831,  0.1644,  0.0559],\n",
       "        ...,\n",
       "        [-0.3427, -0.6770, -0.0309,  ...,  1.3329,  0.8802,  3.4333],\n",
       "        [ 0.3186,  0.6176, -2.1556,  ..., -0.6732,  0.1794,  0.4176],\n",
       "        [-0.4421, -1.5190,  0.4582,  ...,  0.6828, -1.1463, -0.1305]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.embeddings.weight.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "trained_embeddings = trained_model.embeddings.weight.data\n",
    "trained_layer1 = trained_model.linear1.weight.data\n",
    "trained_layer2 = trained_model.linear2.weight.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(torch.eq(embedding_weights, trained_embeddings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(torch.eq(trained_layer1, layer1_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(torch.eq(trained_layer2, layer2_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "bool value of Tensor with more than one value is ambiguous",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-38-ffa35e400479>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0membedding_weights\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mtrained_weights\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"True\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: bool value of Tensor with more than one value is ambiguous"
     ]
    }
   ],
   "source": [
    "if embedding_weights == trained_weights:\n",
    "    print(\"True\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "\n",
    "    main()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
